{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8795b910",
   "metadata": {},
   "source": [
    "# 数据清洗步骤"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "076b70df",
   "metadata": {},
   "source": [
    "## 数据类型转换"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f399df2e",
   "metadata": {},
   "source": [
    "- 利用pandas进行数据处理的时候，经常会遇到数据类型的问题\n",
    "    - 拿到数据后，首先需要确定数据类型是否正确（dtyps），如果不是，需要转化数据类型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8487e878",
   "metadata": {},
   "source": [
    "- pandas 中的数据类型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83ade43b",
   "metadata": {},
   "outputs": [],
   "source": [
    "C:\\Users\\illus\\OneDrive\\A-Teaching\\Data-Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9dc1c38",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 更改数据类型\n",
    "# 1. astype()强制转化数据类型\n",
    "# 2. 通过创建自定义的函数进行数据转化\n",
    "# 3. pandas提供的to_nueric()以及to_datetime()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60d3c97f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Customer Number'].astype(\"int\")\n",
    "#  这样的操作并没有改变原始的数据框，而只是返回的一个拷贝\n",
    "\n",
    "df[\"Customer Number\"] = df[\"Customer Number\"].astype(\"int\")\n",
    "print(df)\n",
    "print(\"--------\"*10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebd5b15b",
   "metadata": {},
   "outputs": [],
   "source": [
    "通过自定义函数清理数据\n",
    "def convert_currency(var):\n",
    "    \"\"\"\n",
    "    convert the string number to a float\n",
    "    _ 去除$\n",
    "    - 去除逗号，\n",
    "    - 转化为浮点数类型\n",
    "    \"\"\"\n",
    "    new_value = var.replace(\",\",\"\").replace(\"$\",\"\")\n",
    "    return float(new_value\n",
    "                 \n",
    "# 通过replace函数将$以及逗号去掉，然后字符串转化为浮点数，让pandas选择pandas认为合适的特定类型，float或者int，该例子中将数据转化为了float64\n",
    "# 通过pandas中的apply函数将2016列中的数据全部转化\n",
    "df[\"2016\"].apply(convert_currency)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24e4e887",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 当然可以通过lambda 函数将这个比较简单的函数一行带过\n",
    "df[\"2016\"].apply(lambda x: x.replace(\",\",\"\").replace(\"$\",\"\")).astype(\"float64\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "c1bf6fab",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'Active'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3360\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3361\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcasted_key\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3362\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\_libs\\index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'Active'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_17984/3817281184.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# 同样可以通过自定义函数进行解决，结果同上\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m# 最后一个自定义函数是利用np.where() function 将Active 列转化为布尔值。\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"Active\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwhere\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"Active\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"Y\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"Active\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3456\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3457\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3458\u001b[1;33m             \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3459\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3460\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3361\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcasted_key\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3362\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3363\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0merr\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3364\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3365\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0misna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhasnans\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'Active'"
     ]
    }
   ],
   "source": [
    "# 同样可以通过自定义函数进行解决，结果同上\n",
    "# 最后一个自定义函数是利用np.where() function 将Active 列转化为布尔值。\n",
    "df[\"Active\"] = np.where(df[\"Active\"] == \"Y\", True, False)\n",
    "\n",
    "df[\"Active\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebd06a4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 利用pandas 函数处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5e6a060",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pandas中pd.to_numeric()处理Jan Units中的数据\n",
    "pd.to_numeric(df[\"Jan Units\"],errors='coerce').fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d2ec6d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 最后利用pd.to_datatime()将年月日进行合并\n",
    "pd.to_datetime(df[['Month', 'Day', 'Year']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96ce7306",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 做到这里不要忘记重新赋值，否则原始数据并没有变化\n",
    "df[\"Jan Units\"] = pd.to_numeric(df[\"Jan Units\"],errors='coerce')\n",
    "df[\"Start_date\"] = pd.to_datetime(df[['Month', 'Day', 'Year']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "497c0bfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将这些转化整合在一起\n",
    "def convert_percent(val):\n",
    "    \"\"\"\n",
    "    Convert the percentage string to an actual floating point percent\n",
    "    - Remove %\n",
    "    - Divide by 100 to make decimal\n",
    "    \"\"\"\n",
    "    new_val = val.replace('%', '')\n",
    "    return float(new_val) / 100\n",
    "\n",
    "df_2 = pd.read_csv(\"sales_data_types.csv\",dtype={\"Customer_Number\":\"int\"},converters={\n",
    "    \"2016\":convert_currency,\n",
    "    \"2017\":convert_currency,\n",
    "    \"Percent Growth\":convert_percent,\n",
    "    \"Jan Units\":lambda x:pd.to_numeric(x,errors=\"coerce\"),\n",
    "    \"Active\":lambda x: np.where(x==\"Y\",True,False)\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c63c82af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选取数据类型\n",
    "\n",
    "# 如果只关注category 类型的数据，其实根本没有必要拿到这些全部数据，只需要将object类型的数据取出，然后进行后续分析即可\n",
    "obj_df = df.select_dtypes(include=['object']).copy()\n",
    "obj_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34633e8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#  在进行下一步处理的之前，需要将数据进行缺失值的处理，对列进行处理axis=1\n",
    "obj_df[obj_df.isnull().any(axis=1)]\n",
    "\n",
    " 处理缺失值的方式有很多种，根据项目的不同或者填补缺失值或者去掉该样本。本文中的数据缺失用该列的众数来补充。\n",
    "obj_df.num_doors.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95ada140",
   "metadata": {},
   "outputs": [],
   "source": [
    "在处理完缺失值之后，有以下几种方式进行category数据转化encoding\n",
    "Find and Replace\n",
    "label encoding\n",
    "One Hot encoding\n",
    "Custom Binary encoding\n",
    "sklearn\n",
    "advanced Approaches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3d0f3f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#  pandas里面的replace文档非常丰富，笔者在使用该功能时候，深感其参数众多，深感提供的功能也非常的强大\n",
    "# 本文中使用replace的功能，创建map的字典，针对需要数据清理的列进行清理更加方便，例如：\n",
    "cleanup_nums= {\n",
    "    \"num_doors\":{\"four\":4,\"two\":2},\n",
    "    \"num_cylinders\":{\n",
    "        \"four\":4,\"six\":6,\"five\":5,\"eight\":8,\"two\":2,\"twelve\":12,\"three\":3\n",
    "    }\n",
    "}\n",
    "obj_df.replace(cleanup_nums,inplace=True)\n",
    "obj_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3fc5df4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 通过pandas里面的 category数据类型，可以很方便的或者该编码\n",
    "obj_df[\"body_style\"]=obj_df[\"body_style\"].astype(\"category\")\n",
    "obj_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e25d3bd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 我们可以通过赋值新的列，保存其对应的code\n",
    "# 通过这种方法可以舒服的数据，便于以后的数据分析以及整理\n",
    "obj_df[\"body_style_code\"] = obj_df[\"body_style\"].cat.codes\n",
    "obj_df.head()## 1. 数据类型转换"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "feb98f06",
   "metadata": {},
   "source": [
    "## 重复值处理"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f6a3a3e",
   "metadata": {},
   "source": [
    "## 缺失值处理"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da94c3a1",
   "metadata": {},
   "source": [
    "## 异常值处理"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ae8438c",
   "metadata": {},
   "source": [
    "## 数据离散化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79c580b4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9812c435",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
